package com.atguigu.flink.chapter06;

import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.KeyedProcessFunction;
import org.apache.flink.util.Collector;

import java.util.HashSet;
import java.util.Set;

/**
 * TODO
 *
 * @author cjp
 * @version 1.0
 * @date 2021/1/22 13:52
 */
public class Flink05_UV {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        env.setParallelism(1);

        // 1.读取数据、转换成 POJO
        SingleOutputStreamOperator<UserBehavior> userBehaviorDS = env
                .readTextFile("input/UserBehavior.csv")
                .map(new MapFunction<String, UserBehavior>() {
                    @Override
                    public UserBehavior map(String value) throws Exception {
                        String[] datas = value.split(",");
                        return new UserBehavior(
                                Long.valueOf(datas[0]),
                                Long.valueOf(datas[1]),
                                Integer.valueOf(datas[2]),
                                datas[3],
                                Long.valueOf(datas[4])
                        );
                    }
                });

        // 2.处理数据: UV => pv行为， 按照 UserID 去重 => Set里面放 UserID ， Set的大小就是 uv值
        userBehaviorDS
                .filter(r -> "pv".equals(r.getBehavior()))
                .map(data -> Tuple2.of("uv", data.getUserId()))  // 构造二元组，"uv"用来分组， userID用来去重
                .returns(Types.TUPLE(Types.STRING,Types.LONG))
                .keyBy(r -> r.f0)
                .process(new KeyedProcessFunction<String, Tuple2<String, Long>, Long>() {
                    // 这个Set在内存里，生产中会借助于redis，存到redis的Set结构中
                    // 如果为了进一步节约内存，可以使用 布隆过滤器
                    private Set<Long> uvCount = new HashSet<>();

                    @Override
                    public void processElement(Tuple2<String, Long> value, Context ctx, Collector<Long> out) throws Exception {
                        // 把用户ID存到Set里
                        uvCount.add(value.f1);
                        out.collect((long)uvCount.size());
                    }
                })
                .print();


        env.execute();
    }
}
